# relations_helper.pl
use strict;
use warnings;

require 'csv.pl';
require 'jsonpath.pl';

require '_items_helper.pl';
require '_items_index_helper.pl';

my $FALSE = 0;
my $TRUE  = 1;

my $TRANSFORM_MANIFEST_FILE = '_transform.js';



sub helper_creating {
    #
    # Tests whether the create_from manifest file is still in the relation's folder.
    # This is a reliable way of testing whether transformer.pl (on cron) is still copying
    # and transforming items into the folder. Once completed the transformer.pl deletes the file.
    #
    # Returns $TRUE if creating items; $FALSE otherwise.
    #
    my ($folder_type, $settings, $params) = @_;
    
    my $folder_path = helper_folders_path($folder_type, $settings, $params);
    if (performed_render()) {
        return($FALSE);
    }

    my $create_from_file = File::Spec->catfile($folder_path, $TRANSFORM_MANIFEST_FILE);
    
    return (-e $create_from_file);
}


sub helper_create_transform_task {
    #
    #   create manifest listing folders (and their items) to transform
    #
    #   Return: 1=finished, 0=queued, undef=aborted
    #
    my ($folder_type, $settings, $params) = @_;

    my $is_output = $FALSE;
    my $folder_info = helper_folders_get_info($folder_type, $settings, $params, $is_output);
    if (performed_render()) {
        return;
    }

    my $folder_path = helper_folders_path($folder_type, $settings, $params);
    if (performed_render()) {
        return;
    }

    # construct transform manifest
    my $transform_manifest = {};

    # get the length of each argument folder
    my @argument_size = ();
    my @argument_index = ();
    for my $argument_folder_id (@{$folder_info->{'relation_ids'}}) {
        my $argument_path = helper_folders_path($folder_type, $settings, $params, $argument_folder_id);
        if (performed_render()) {
            return;
        }
        my $size = helper_items_index_size($argument_path);
        push(@argument_size, $size);
        push(@argument_index, 0);
        if ($folder_info->{'generator'} eq 'map' || $folder_info->{'generator'} eq 'lambda') {
            # all arguments must have same number of items
            if ($size != $argument_size[0]) {
                render({
                    'status' => '400 Bad Request',
                    'text'   => serialise_error_message("Relations must have the same number of items."),
                });
                return;
            }
        }
    }
    $transform_manifest->{'size'} = \@argument_size;
    $transform_manifest->{'index'} = ($folder_info->{'generator'} eq 'map' || $folder_info->{'generator'} eq 'lambda') ? [0] : \@argument_index;

    # save manifest into the results folder
    my $transform_file = File::Spec->catfile($folder_path, $TRANSFORM_MANIFEST_FILE);
    util_writeFile($transform_file, JSON->new->canonical->pretty->encode($transform_manifest) );   

}


sub helper_transform_next {
    #
    #   helper_transform_next($folder_type, $settings, $params[, $folder_info])
    #
    #   Incrementally copy-transform items from source to destination folder
    #

    my ($folder_type, $settings, $params, $folder_info) = @_;

    my $start_time = time;

    # only enact for a maximum of N seconds (because running via cron)
    my $TRANSFORM_TIMEOUT = $settings->{'TRANSFORM_TIMEOUT'} || 10;

    # get the metadata of the results folder
    if (!defined $folder_info) {
        my $is_output = $FALSE;
        $folder_info = helper_folders_get_info($folder_type, $settings, $params, $is_output);
        if (performed_render()) {
            return;
        }
    }

    my $folder_path = helper_folders_path($folder_type, $settings, $params);
    if (performed_render()) {
        return;
    }

    my $transform_file = File::Spec->catfile($folder_path, $TRANSFORM_MANIFEST_FILE);
    if (!-e $transform_file) {
        render({
            'status' => '404 Not Found',
            'text'   => serialise_error_message("Nothing to run."),
        });
        return $TRUE; #finished
    }    
    my $transform_manifest = JSON->new->decode( util_readFile($transform_file) );

#    my @argument_folders_info = ();
    my @argument_folders_value = ();
    my @argument_folders_path = ();
    for my $argument_folder_id (@{$folder_info->{'relation_ids'}}) {

        my $is_output = $FALSE;
#my $is_output = $TRUE;      #DELETE ME!!!!!!!!!!!!!!!!!!
        my $argument_folder_info = helper_folders_get_info($folder_type, $settings, $params, $is_output, $argument_folder_id);
        if (performed_render()) {
            return;
        }
#        push(@argument_folders_info, $argument_folder_info);
        push(@argument_folders_value, $argument_folder_info->{'value'});

        my $argument_folder_path = helper_folders_path($folder_type, $settings, $params, $argument_folder_id);
        if (performed_render()) {
            return;
        }
        push(@argument_folders_path, $argument_folder_path);
    }

    my $size = $transform_manifest->{'size'};

    my $generator = $folder_info->{'generator'};

    my $compiled_templates = compile_template($folder_info->{'templates'});
    if (performed_render()) {
        return;
    }

    my $context = {
        'relation'    => $folder_info->{'value'} || {},
#        'item'      => {} or [] or, as extension to JSON standard, scalars
        'item_id_i' => helper_items_index_size($folder_path), # auto id counter (intended for use in lambda generators)
        'relations'   => \@argument_folders_value,
#        'items'     => [],
         'arguments_size' => $transform_manifest->{'size'},
    };
    # make context available to jm: operators (e.g. for jm:generate template evaluation)
    context_set($context);
    # optionally, run any initialisation (e.g. register external web service functions with jm:register)
    if (exists $compiled_templates->{'init'}) {
        evaluate_template_value($compiled_templates->{'init'}, $context)
    }
    # if target value already has data in it, merge new values into existing ones
    $folder_info->{'value'} = ( 
        (!defined $folder_info->{'value'})
         ? evaluate_template_value($compiled_templates->{'relation'}, $context)
         : _multiset_union(
             $folder_info->{'value'}, 
             evaluate_template_value($compiled_templates->{'relation'}, $context)
             #NB. multiset_union assumes that we're dealing with hashes or even-sized arrays
        )
    );
    $context->{'relation'} = $folder_info->{'value'};

    #
    # process folder item transformations until finished or timeout
    #
    
    my $finished = $FALSE;
    LP_MANIFEST:
    for (;;) {
        # only execute for a maximum of N seconds (+ time of one iteration)
        if ((time - $start_time) > $TRANSFORM_TIMEOUT) {
            last LP_MANIFEST;
        }

        # load arguments at current index
        my $index = $transform_manifest->{'index'};
        my $argument_values = ();
        #FIXME: optimisation to batch load, say, 1000 ids at a time rather than single id queries (at least for innermost index)
        #FIXME: optimisation for reflexive comparisons (no need to reload the same data multiple times)
        #FIXME: optimisation for symmetrical operators in a 'product' relationship where only need calc half of matrix
        my @item_ids = ();
        my @item_values = ();
        for(my $i=0; $i < scalar(@argument_folders_path); $i++) {
            my $ix = ($generator eq 'map' || $generator eq 'lambda') 
                     ?  $index->[0] 
                     :  $index->[$i];
            my $item_ids = helper_items_index_get($argument_folders_path[$i], 1+$ix, 1);
            my $is_output = $FALSE;
            my $value = helper_items_get($folder_type, $settings, $params, @$item_ids[0], $is_output, $folder_info->{'relation_ids'}[$i]);
            push(@item_ids, @$item_ids[0]);
            push(@item_values, $value);
        }

        # apply function to arguments and, for map/produce (not lambda), store value as a new item

        my $item_id;
        if    ($generator eq 'map')     { $item_id = $item_ids[0]; }
        elsif ($generator eq 'product') { $item_id = join('_x_', @item_ids); }
        # else generator eq 'lambda' => leave $item_id as undef

        $context->{'item_ids'} = \@item_ids;
        $context->{'items'} = \@item_values;

        my $generated_items = {};
        my $exclude_item = $FALSE;
        try {
            if (exists $compiled_templates->{'restrict'} && 
                !evaluate_template_value($compiled_templates->{'restrict'}, $context)
               ) {
                $exclude_item = $TRUE;
            }
            else {
                $context->{'item_id'} = $item_id;   # make id available during evaluation (overridden in lambda generators)
                $context->{'item'} = undef;         # note: in lambda generators, the old value of an item can be fetched
                generate_set($generated_items);
                # evaluate template (result is meaningless in lambda generator because generates a result set)
                my $result = evaluate_template_value(
                    $compiled_templates->{'item'},
                    $context
                );
                $generated_items = ($generator eq 'lambda') ? generate_get() : {$item_id => $result};
                if ($compiled_templates->{'item_each_item'}) {
                    # evaluate item_each_item template for each item generated
                    while( my ($id, $value) = each(%$generated_items) ) {
                        $context->{'item_id'} = $id;
                        $context->{'item'} = $value;
                        $generated_items->{$id} = evaluate_template_value(
                            $compiled_templates->{'item_each_item'}, 
                            $context,
                            $value
                        );
                    }
                }
                if ($compiled_templates->{'relation_each_item'}) {
                    # evaluate folder_each_item template for each item generated
                    while( my ($id, $value) = each(%$generated_items) ) {
                        $context->{'item_id'} = $id;
                        $context->{'item'} = $value;
                        $folder_info->{'value'} = evaluate_template_value(
                            $compiled_templates->{'relation_each_item'}, 
                            $context,
                            $context->{'relation'}
                        );
                        $context->{'relation'} = $folder_info->{'value'};
                    }
                }
            }
        } catch {
            # already rendered error inside helper_apply_json_path
            error_message("Error evaluating template: $_");
            # abort transformation to avoid thousands of errors
            unlink($transform_file);
            return;
        };

        if (!$exclude_item) {
            # create one (for map/product) or more persistent items (for lambda)
            # NB. let helper do the check for an empty hash to avoid executing perl 'keys' function twice (in case hash is large)
            helper_items_create($folder_type, $settings, $params, $generated_items);
            if (performed_render()) {
                return;
            }
        }

#print STDERR "WARNING -- DUBUG LOOP QUIT AFTER 1 ITER\n"; $finished=1;last LP_MANIFEST; 
        # increment rightmost index
        LP_INCREMENT_INDICES:
        for(my $i=scalar(@$index)-1; $i >= 0; $i--) {
            $index->[$i]++;
            if ($index->[$i] < $size->[$i]) {
                last LP_INCREMENT_INDICES;
            }
            else {
                if ($i == 0) {
                    $finished = $TRUE;
                    last LP_MANIFEST;
                }
                $index->[$i] = 0;
            }
        }
#print STDERR "WARNING -- DUBUG QUIT LP\n"; last LP_MANIFEST; 
    }

    # update state information held at the folder level (i.e. in $.value field)
    helper_folders_put_info($folder_type, $settings, $params, $folder_info);
    if (performed_render()) {
        return;
    }

    if ($finished) {
        #
        # finished all tranformations, so delete the transform file
        #
        unlink($transform_file);
    }
    else {
        #
        # preserve state for subsequent continuation
        #
        util_writeFile($transform_file, JSON->new->canonical->pretty->encode($transform_manifest) );
        #
        # create a workflow task to run the transformation on cron job
        #
        my $folder_id = $folder_info->{'id'};
        my $workflow_id = '_' . $folder_id;
        my $commands = "*, post, relations/$folder_id/run";
        require 'system_helper.pl';
        helper_create_workflow_task($params, $settings, $workflow_id, $commands);
    }

    return $finished;
}

# local copy of helper_fn_multiset_union so that don't have dependencies on jm: functions in fn_helper.pl file
sub _multiset_union {
    #
    #   HASH_REF multiset_union($multiset1, $multiset2, ...)
    #
    #   merges a list of multisets into a single multiset (i.e. may change the first arg if it is a multiset)
    #   sums the multiplicities of corresponding keys
    #   returns multiset <STRING term, INT multiplicity>
    #
    #   unpacks any array arguments to lists of key,value, which are processed as a hash
    #   scalars are treated as if a singleton multiset whose key is the scalar and multiplicity is 1
    #
    my $hash_ref;
    LP_ARGS:
    foreach my $arg (@_) {
        # skip null arguments
        if (!defined $arg) {
            next LP_ARGS;
        }
        if (!ref($arg)) {
            # treat scalars as a singleton multiset
            $arg = [$arg, 1];
        }
        if (ref($arg) eq 'ARRAY') {
            if (!defined $hash_ref) {
                # create a new hash from first argument
                my %hash = @$arg;
                $hash_ref = \%hash;
            }
            else {
                # merge into destination hash
                while(scalar @$arg) {
                    my $key = shift @$arg;
#$key = Encode::decode('UTF-8', $key);
                    my $value = shift @$arg;
                    if (!exists $hash_ref->{$key}) {
                        $hash_ref->{$key} = $value;
                    }
                    else {
                        $hash_ref->{$key} += $value;
                    }
                }
            }
        }
        elsif (ref($arg) eq 'HASH') {
            if (!defined $hash_ref) {
                # first multiset argument becomes the destination hash
                $hash_ref = $arg;
            }
            else {
                # merge into destination hash
                while( my ($key, $value) = each(%$arg) ) {
#$key = Encode::decode('UTF-8', $key);
                    if (!exists $hash_ref->{$key}) {
                        $hash_ref->{$key} = $value;
                    }
                    else {
                        $hash_ref->{$key} += $value;
                    }
                }
            }
        }
    }
    return $hash_ref || {};
}


1;